import scrapy
from scrapy import Selector
from scrapy.http import HtmlResponse

from Visualization.items import CityItem


class CitySpider(scrapy.Spider):
    name = "city"
    allowed_domains = ["www.tianqi.com"]
    start_urls = ["https://www.tianqi.com/chinacity.html"]

    def parse(self, response:HtmlResponse):
        sel = Selector(response)
        slt="body > div.wrap1100 > div.wrapbox > div.hotcity > span > a"
        cities=sel.css(slt)
        for list_item in cities:
            city_item = CityItem()
            href = list_item.css('::attr(href)').get()
            text = list_item.css('::text').get()
            city_item['name'] = text
            city_item['city_href'] = response.urljoin(href)
            print(f'City: {text}, URL: {href}')
            yield city_item